package com.yangit.temp.common.controller;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVPrinter;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.junit.Test;
import org.springframework.stereotype.Controller;


import com.yangit.temp.common.util.file.HtmlParser;
import com.yangit.temp.common.util.http.HttpClientUtil2;

@Controller
public class HtmlParseController {

	
	public static void main(String[] args) throws IOException {
		
		HtmlParseController htmlParseController = new HtmlParseController();
		String sc = "41";
		String[] ccs= {"07","08","09","10","11","12","13","14","15","16","17","90"};
		//String[] ccs= {"01"};
		for(int i=0;i<ccs.length;i++) {
			String cc = ccs[i];
			String url = sc+"/"+sc+cc; // 41/4010.html
			String html = HttpClientUtil2.doGet("http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/"+url+".html");
			List<String> list = htmlParseController.getshiquList(html);
			
			for (int j = 0; j < list.size(); j++) {
				String urlxiangzhen = sc+"/"+cc+"/"+(list.get(j)).substring(0,6); // 41/01/410102.html
				System.out.println(urlxiangzhen);
				String htmlxiangzhen = HttpClientUtil2.doGet("http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/"+urlxiangzhen+".html");
				List<String> listjiedao = htmlParseController.getjiedaoList(htmlxiangzhen);
				System.out.println("listjiedao:"+listjiedao.toString());
				for (int k = 0; k < listjiedao.size(); k++) {
					String urljiedao = sc+"/"+cc+"/"+(list.get(j)).substring(4,6)+"/"+listjiedao.get(k).substring(0,9); // 41/01/410102.html
					System.out.println("urljiedao:"+urljiedao);
					String htmljiedao = HttpClientUtil2.doGet("http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/"+urljiedao+".html");
					htmlParseController.getxaingzhen(htmljiedao);
				}
				
			}
			
		}
	}
	

	
	public  void getxaingzhen(String html) throws IOException{

		
		
		System.out.println("res:"+html);
		//6.Jsoup解析html
        Document document = Jsoup.parse(html);
		List<List<String>> tables = HtmlParser.getTables(document);
		System.out.println("____________________________________________________________");
		
		System.out.println(tables);
		System.out.println("____________________________________________________________");
		List<String> table =tables.get(1);
		System.out.println(table);
		System.out.println(table.size());

        FileOutputStream fos = new FileOutputStream("C:\\Users\\Administrator\\Desktop\\out.csv",true);
        OutputStreamWriter osw = new OutputStreamWriter(fos, "GBK");

        CSVFormat csvFormat = CSVFormat.DEFAULT.withHeader("城乡分类", "名称");
        CSVPrinter csvPrinter = new CSVPrinter(osw, csvFormat); 
		for (int i = 7; i < table.size()-1;i++) {
			String reString =table.get(i);
			System.out.println(i+":"+reString);
			reString =table.get(i).substring(0,6)+","+table.get(i+2);
			csvPrinter.printRecord(reString);      
			i=i+2;
			System.out.println(i+":"+reString);
		}



        csvPrinter.flush();
	    csvPrinter.close();
	
	}
	/**
	 * 获取区
	 * @param html
	 * @throws IOException
	 */
	public  void getshiqu(String html) throws IOException{

		
		
		System.out.println("res:"+html);
		//6.Jsoup解析html
        Document document = Jsoup.parse(html);
		List<List<String>> tables = HtmlParser.getTables(document);
		System.out.println("____________________________________________________________");
		
		System.out.println(tables);
		System.out.println("____________________________________________________________");
		List<String> table =tables.get(1);
		System.out.println(table);
		System.out.println(table.size());

        FileOutputStream fos = new FileOutputStream("C:\\Users\\Administrator\\Desktop\\out.csv",true);
        OutputStreamWriter osw = new OutputStreamWriter(fos, "GBK");

        CSVFormat csvFormat = CSVFormat.DEFAULT.withHeader("区划代码", "城乡分类");
        CSVPrinter csvPrinter = new CSVPrinter(osw, csvFormat); 
		for (int i = 8; i < table.size()-1;i++) {
			String reString =table.get(i);
			System.out.println(i+":"+reString);
			reString =table.get(i).substring(0,6)+","+table.get(i+1);
			csvPrinter.printRecord(reString);      
			i=i+1;
			System.out.println(i+":"+reString);
		}

        csvPrinter.flush();
	    csvPrinter.close();
	
	}
	
	/**
	 * -获取区
	 * @param html
	 * @throws IOException
	 */
	public  List<String> getshiquList(String html) throws IOException{
		
		
		List<String> list  = new ArrayList<String>();
		System.out.println("res:"+html);
		//6.Jsoup解析html
		Document document = Jsoup.parse(html);
		List<List<String>> tables = HtmlParser.getTables(document);
		System.out.println("____________________________________________________________");
		
		System.out.println(tables);
		System.out.println("____________________________________________________________");
		List<String> table =tables.get(1);
		System.out.println(table);
		System.out.println(table.size());
		
		FileOutputStream fos = new FileOutputStream("C:\\Users\\Administrator\\Desktop\\out.csv",true);
		OutputStreamWriter osw = new OutputStreamWriter(fos, "GBK");
		
		CSVFormat csvFormat = CSVFormat.DEFAULT.withHeader("区划代码", "城乡分类");
		CSVPrinter csvPrinter = new CSVPrinter(osw, csvFormat); 
		for (int i = 8; i < table.size()-1;i++) {
			String reString =table.get(i);
			System.out.println(i+"市区:"+reString);
			reString =table.get(i).substring(0,6)+","+table.get(i+1);
			list.add(table.get(i));
			csvPrinter.printRecord(reString);      
			i=i+1;
			System.out.println(i+":"+reString);
		}
		csvPrinter.flush();
	    csvPrinter.close();	
		return list;
		
	}
	

	/**
	 * -获取街道
	 * @param html
	 * @throws IOException
	 */
	public  List<String> getjiedaoList(String html) throws IOException{
		
		
		List<String> list  = new ArrayList<String>();
		System.out.println("res:"+html);
		//6.Jsoup解析html
		Document document = Jsoup.parse(html);
		List<List<String>> tables = HtmlParser.getTables(document);
		System.out.println("____________________________________________________________");
		
		System.out.println(tables);
		System.out.println("____________________________________________________________");
		List<String> table =tables.get(1);
		System.out.println(table);
		System.out.println(table.size());
		
		FileOutputStream fos = new FileOutputStream("C:\\Users\\Administrator\\Desktop\\out.csv",true);
		OutputStreamWriter osw = new OutputStreamWriter(fos, "GBK");
		
		CSVFormat csvFormat = CSVFormat.DEFAULT.withHeader("区划代码", "城乡分类");
		CSVPrinter csvPrinter = new CSVPrinter(osw, csvFormat); 
		for (int i = 6; i < table.size()-1;i++) {
			String reString =table.get(i);
			System.out.println("街道00----00:"+reString);
			reString =table.get(i).substring(0,6)+","+table.get(i+1);
			list.add(table.get(i));
			csvPrinter.printRecord(reString);      
			System.out.println(i+"街道:"+reString);
			i=i+1;
		}
		csvPrinter.flush();
		csvPrinter.close();	
		return list;
		
	}


	public static byte[] getUTF8BytesFromGBKString(String gbkStr) {  
	    int n = gbkStr.length();  
	    byte[] utfBytes = new byte[3 * n];  
	    int k = 0;  
	    for (int i = 0; i < n; i++) {  
	        int m = gbkStr.charAt(i);  
	        if (m < 128 && m >= 0) {  
	            utfBytes[k++] = (byte) m;  
	            continue;  
	        }  
	        utfBytes[k++] = (byte) (0xe0 | (m >> 12));  
	        utfBytes[k++] = (byte) (0x80 | ((m >> 6) & 0x3f));  
	        utfBytes[k++] = (byte) (0x80 | (m & 0x3f));  
	    }  
	    if (k < utfBytes.length) {  
	        byte[] tmp = new byte[k];  
	        System.arraycopy(utfBytes, 0, tmp, 0, k);  
	        return tmp;  
	    }  
	    return utfBytes;  
	}
	public static String getUTF8StringFromGBKString(String gbkStr) {  
	    try {  
	        return new String(getUTF8BytesFromGBKString(gbkStr), "UTF-8");  
	    } catch (UnsupportedEncodingException e) {  
	        throw new InternalError();  
	    }  
	}

	/**
	 * 将编码为srcCharset的字节数组src转化为编码为destCharet的字节数组
	 * 
	 * @param src
	 * @param srcCharset
	 * @param destCharet
	 * @return
	 * @throws UnsupportedEncodingException
	 */
	public static byte[]  convertEncoding_ByteArr(byte[] src,String srcCharset,String destCharet) 
			throws UnsupportedEncodingException{	
		String s = new String(src, srcCharset);
		return s.getBytes(destCharet);
	}
	
	
	public static int convertFileEncoding(String srcFilePath,String srcCharset ,
			String destFilePath,String destCharset,boolean isDeleteSrc) throws  IOException {
		if(srcFilePath == null || srcFilePath.length() == 0)
			throw new IllegalArgumentException("srcFilePath is empty.");
		if(destFilePath == null || destFilePath.length() == 0)
			throw new IllegalArgumentException("destFilePath is empty.");
		if(srcFilePath.equalsIgnoreCase(destFilePath))
			throw new IllegalArgumentException("srcFilePath is the same as destFilePath");
		
		if(srcCharset == null || srcCharset.length() == 0)
			throw new IllegalArgumentException("srcCharset is empty.");
		if(destCharset == null || destCharset.length() == 0)
			throw new IllegalArgumentException("destCharset is empty.");
		
		if(srcCharset.equalsIgnoreCase(destCharset)) // 编码相同，无需转换
			return 0;
		
		File srcFile = new File(srcFilePath);
		
		FileInputStream fis = null;
		InputStreamReader isr = null;
		BufferedReader br = null;
		
		FileOutputStream fos = null;
		OutputStreamWriter osw = null;

		try {
			fis = new FileInputStream(srcFile);
			isr = new InputStreamReader(fis, srcCharset);
			
			// BufferedReader中defaultCharBufferSize = 8192.
			// 即：8192 × 2 byte = 16k
			// 若是utf-8,中文占3个字节，16K / 3  = 5461，即只要每行中文字符数 < 5461,读取的行数就是准确的，
			// 否则，可能会截断一行，多写入'\n',但这种情况一般不存在。
			// 如果源文件中最后一行没有换行符，转码后的文件最后会多写入一个换行符
			br = new BufferedReader(isr);

			// 以UTF-8格式写入文件,file.getAbsolutePath()即该文件的绝对路径,false代表不追加直接覆盖,true代表追加文件
			fos = new FileOutputStream(destFilePath, false);
			osw = new OutputStreamWriter(fos, destCharset);

			String str = null;

			// 创建StringBuffer字符串缓存区
			StringBuffer sb = new StringBuffer();
			int lines = 0;

			// 通过readLine()方法遍历读取文件
			while ((str = br.readLine()) != null) {
				// 使用readLine()方法无法进行换行,需要手动在原本输出的字符串后面加"\n"或"\r"
				sb.append(str).append('\n');
				osw.write(sb.toString());
				osw.flush();				
				lines++;
			}
			
			if(isDeleteSrc) 
			{
				if(srcFile.delete())
					System.out.println(srcFile.getAbsolutePath() + " file is already deleted.");
				else
					System.out.println(srcFile.getAbsolutePath() + " file delete fail.");
			}
			
//			System.out.println(lines);
			return lines;
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
//			e.printStackTrace();
			throw e;
		} catch (UnsupportedEncodingException e) {
			// TODO Auto-generated catch block
			throw e;
		} catch (IOException e) {
			// TODO: handle exception
			throw e;
		} finally {
			// 与同一个文件关联的所有输出流(输入流)，只需关闭一个即可
			if (null != fis)
				try {
					fis.close();
					fis = null;
				} catch (IOException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}

			if (null != fos)
				try {
					fos.close();
					fos = null;
				} catch (IOException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
		}
	}

	public static String utf8Togb2312(String str){

        StringBuffer sb = new StringBuffer();

        for ( int i=0; i<str.length(); i++) {

            char c = str.charAt(i);

            switch (c) {

               case '+' :

                   sb.append( ' ' );

               break ;

               case '%' :

                   try {

                        sb.append(( char )Integer.parseInt (

                        str.substring(i+1,i+3),16));

                   }

                   catch (NumberFormatException e) {

                       throw new IllegalArgumentException();

                  }

                  i += 2;

                  break ;

               default :

                  sb.append(c);

                  break ;

             }

        }

        String result = sb.toString();

        String res= null ;

        try {

             byte [] inputBytes = result.getBytes( "8859_1" );

            res= new String(inputBytes, "UTF-8" );

        }

        catch (Exception e){}

        return res;

  }

// 将 GB2312 编码格式的字符串转换为 UTF-8 格式的字符串：

public static String gb2312ToUtf8(String str) {

        String urlEncode = "" ;

        try {

            urlEncode = URLEncoder.encode (str, "UTF-8" );

        } catch (UnsupportedEncodingException e) {

            e.printStackTrace();

        }

        return urlEncode;

}
}
